import asyncio

from loguru import logger

from com.arcfox.base.base_spider import ExecuteType
from com.arcfox.manager.redis_task_manager import RedisTaskManager
from com.arcfox.school.parse.jzy_major_parse import MajorParser

from com.arcfox.school.parse.yifen_yiduan_parse import YifenYiduanParser
from com.arcfox.school.spider.jzy.jzy_base_spider import JZYBaseSpider
from com.arcfox.util import async_request as requests, util
from com.arcfox.util.util import random_sleep

'''
    一分一段
'''


class YifenYiduanSpider(JZYBaseSpider):
    def __init__(self):
        super().__init__()
        self.start_url = "http://www.juzhiyuan.com/yifenyiduan?rand=85&province=34"
        # self.processor = MajorProcessor()
        self.parser = YifenYiduanParser()
        self.task_manager = RedisTaskManager(self.TASK_SEGMENT_LIST_JZY)

    async def _pull_task(self):
        return True, ExecuteType.ONCE

    async def _crawl_by_task(self, tasks):
        await self.init_session()
        segment_list = self.parser.parse_yifen_yiduan_list(self.get_province_map())
        if len(segment_list) == 0:
            logger.info("未获取到一分一段信息")
            return
        logger.info("一分一段信息获取数量：{}", len(segment_list))
        result_list = []
        for segment in segment_list:
            try:
                resp = await requests.get(self.session, url=segment.get("url"), headers=self.get_default_header())
                if resp.code == 200:
                    await util.random_sleep(1)
                    result = self.parser.parse_yifen_yiduan_detail_list(segment, resp.response)
                    result_list += result
            except Exception as e:
                logger.error("命中风控, 休眠10分钟:{}",e)
                await random_sleep(10 * 60)
        await self.task_manager.add_tasks(result_list)


if __name__ == "__main__":
    asyncio.run(YifenYiduanSpider().start_crawl())
